Add LearningModelBuilder to WinML Experimental Namespace along with various Audio operators by smk2007 · Pull Request #6623 · microsoft/onnxruntime

smk2007 · 2021-02-09T20:46:47Z

Add LearningModelBuilder to WinML Experimental Namespace along with various Audio operators

This PR includes the following features:

A bunch of experimental Signal Processing Operators to enable Audio scenarios.
- DFT
- IDFT
- STFT
- HannWindow
- HammingWindow
- BlackmanWindow
- MelWeightMatrix
  These operators are included behind the build flag --experimental, and are part of the com.microsoft.experimental domain.
Add experimental ModelBuilding classes to Microsoft.AI.MachineLearning.Expermental namespace
- LearningModelBuilder
- LearningModelOperator
- LearningModelOperatorSet
- LearningModelInputs
- LearningModelOutputs
  These classes enable dynamic onnx model building useful for featurization and testing.

winml/test/api/LearningModelSessionAPITest.cpp

tiagoshibata · 2021-02-10T23:37:10Z

onnxruntime/contrib_ops/cpu/signal/dft.cc

+static bool is_power_of_2(size_t size) {
+  unsigned n_bits = 0;
+  while (size != 0) {
+    n_bits += size & 1;
+    size = size >> 1;
+  }
+  return n_bits == 1;
+}
+
+static const unsigned char BitReverseTable256[] =
+{
+    0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
+    0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
+    0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
+    0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
+    0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
+    0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
+    0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
+    0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
+    0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
+    0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
+    0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
+    0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
+    0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
+    0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
+    0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
+    0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF};
+
+template <unsigned TSignificantBits>
+uint32_t bit_reverse(uint32_t num) {
+  uint32_t rev = (BitReverseTable256[num & 0xff] << 24) |
+         (BitReverseTable256[(num >> 8) & 0xff] << 16) |
+         (BitReverseTable256[(num >> 16) & 0xff] << 8) |
+         (BitReverseTable256[(num >> 24) & 0xff]);
+  return static_cast<uint32_t>(((uint64_t)rev) >> (32 - TSignificantBits));
+}
+
+template <typename T>
+static inline T bit_reverse(T num, unsigned significant_bits) {
+  switch (significant_bits) {
+    case 0: return static_cast<T>(bit_reverse<0>(static_cast<uint32_t>(num)));
+    case 1: return static_cast<T>(bit_reverse<1>(static_cast<uint32_t>(num)));
+    case 2: return static_cast<T>(bit_reverse<2>(static_cast<uint32_t>(num)));
+    case 3: return static_cast<T>(bit_reverse<3>(static_cast<uint32_t>(num)));
+    case 4: return static_cast<T>(bit_reverse<4>(static_cast<uint32_t>(num)));
+    case 5: return static_cast<T>(bit_reverse<5>(static_cast<uint32_t>(num)));
+    case 6: return static_cast<T>(bit_reverse<6>(static_cast<uint32_t>(num)));
+    case 7: return static_cast<T>(bit_reverse<7>(static_cast<uint32_t>(num)));
+    case 8: return static_cast<T>(bit_reverse<8>(static_cast<uint32_t>(num)));
+    case 9: return static_cast<T>(bit_reverse<9>(static_cast<uint32_t>(num)));
+    case 10: return static_cast<T>(bit_reverse<10>(static_cast<uint32_t>(num)));
+    case 11: return static_cast<T>(bit_reverse<11>(static_cast<uint32_t>(num)));
+    case 12: return static_cast<T>(bit_reverse<12>(static_cast<uint32_t>(num)));
+    case 13: return static_cast<T>(bit_reverse<13>(static_cast<uint32_t>(num)));
+    case 14: return static_cast<T>(bit_reverse<14>(static_cast<uint32_t>(num)));
+    case 15: return static_cast<T>(bit_reverse<15>(static_cast<uint32_t>(num)));
+    case 16: return static_cast<T>(bit_reverse<16>(static_cast<uint32_t>(num)));
+    case 17: return static_cast<T>(bit_reverse<17>(static_cast<uint32_t>(num)));
+    case 18: return static_cast<T>(bit_reverse<18>(static_cast<uint32_t>(num)));
+    case 19: return static_cast<T>(bit_reverse<19>(static_cast<uint32_t>(num)));
+    case 20: return static_cast<T>(bit_reverse<20>(static_cast<uint32_t>(num)));
+    case 21: return static_cast<T>(bit_reverse<21>(static_cast<uint32_t>(num)));
+    case 22: return static_cast<T>(bit_reverse<22>(static_cast<uint32_t>(num)));
+    case 23: return static_cast<T>(bit_reverse<23>(static_cast<uint32_t>(num)));
+    case 24: return static_cast<T>(bit_reverse<24>(static_cast<uint32_t>(num)));
+    case 25: return static_cast<T>(bit_reverse<25>(static_cast<uint32_t>(num)));
+    case 26: return static_cast<T>(bit_reverse<26>(static_cast<uint32_t>(num)));
+    case 27: return static_cast<T>(bit_reverse<27>(static_cast<uint32_t>(num)));
+    case 28: return static_cast<T>(bit_reverse<28>(static_cast<uint32_t>(num)));
+    case 29: return static_cast<T>(bit_reverse<29>(static_cast<uint32_t>(num)));
+    case 30: return static_cast<T>(bit_reverse<30>(static_cast<uint32_t>(num)));
+    case 31: return static_cast<T>(bit_reverse<31>(static_cast<uint32_t>(num)));
+    case 32: return static_cast<T>(bit_reverse<32>(static_cast<uint32_t>(num)));
+    default: ORT_THROW("Unsupported bit size.");
+  }
+}


Using compiler intrinsics might be worth it

Which intrinsic?

popcnt for population count (number of 1 bits), rbit to reverse bits in ARM. For bit reverse in x86 there's fancy ways using SIMD that are reportedly >5 times faster than the lookup table solution, like https://stackoverflow.com/a/24058332/1390418 and https://stackoverflow.com/a/46318399/1390418, but it's much trickier to implement

onnxruntime/contrib_ops/cpu/signal/dft.cc

tools/ci_build/build.py

winml/lib/Api.Experimental/LearningModelBuilder.cpp

onnxruntime/contrib_ops/cpu/signal/window_functions.cc

martinb35 · 2021-02-11T02:28:20Z

onnxruntime/contrib_ops/cpu/signal/window_functions.cc

+
+template <typename T>
+static T get_scalar_value_from_tensor(const Tensor* tensor) {
+  ORT_ENFORCE(tensor->Shape().Size() == 1, "ratio input should have a single value.");


is this error message descriptive enough to know which operator is wrong and how to fix it?

Changed to Tensor input should have single value.

The error message can be better here.

onnxruntime/contrib_ops/cpu/signal/window_functions.cc

Sheil Kumar added 30 commits February 27, 2020 16:55

model building

dcc0902

fix build

5130d15

winml adapter model building api

be89b60

model building

10f924c

merge master

af75567

make build

3024363

merge changes

fa5fed6

make build again

58d609e

add model building with audio op

19071ad

inplace and inorder fft

6d6bea1

add ifft

605752e

works!

5a7229e

cleanup

09f4d27

add comments

d4a330e

switch to iterative rather than recursive and use parallelization

02a8a11

batched parallelization

34da4b7

fft->dft

377a107

cleanup

32d0040

window functions

090b602

add melweightmatrix op

c8c6fef

updates to make spectrogram test work

d58b09f

push latest

23076ed

add onesided

17c1574

cleanup

156459d

Clean up building apis and fix mel

96f7cf9

cleanup

bc300f8

cleanup

c97153c

Merge branch 'master' into user/sheilk/training-explorations

33d480d

naive stft

992328b

fix test output

fbe9e4b

Sheil Kumar added 10 commits February 2, 2021 12:14

Merge branch 'master' into user/sheilk/training-explorations

bf9e006

add experimental flag

2d2cbaf

merge

493322d

add experimental api into nuget package

4589c0b

add experimental api build flag and add to windows ai nuget package

135a8b2

turn experimental for tests

29a8ccc

add minimum opset version to new experimental domain

1e42baf

api cleanup

cecf073

Merge branch 'master' into user/sheilk/training-explorations

50a5776

Merge branch 'master' into user/sheilk/training-explorations

3420548

smk2007 requested review from martinb35, orilevari, ryanlai2, tiagoshibata and zhangxiang1993 February 9, 2021 20:46

smk2007 requested a review from a team as a code owner February 9, 2021 20:46

ryanlai2 reviewed Feb 9, 2021

View reviewed changes

winml/test/api/LearningModelSessionAPITest.cpp Outdated Show resolved Hide resolved

Sheil Kumar added 3 commits February 10, 2021 11:08

disable ms experimental ops test when --ms_experimental is not enabled

5c1225c

add macro behind flag

17d1c8b

remove unused x

ebed3bd

tiagoshibata reviewed Feb 10, 2021

View reviewed changes

onnxruntime/contrib_ops/cpu/signal/dft.cc Show resolved Hide resolved

ryanlai2 reviewed Feb 11, 2021

View reviewed changes

tools/ci_build/build.py Outdated Show resolved Hide resolved

ryanlai2 reviewed Feb 11, 2021

View reviewed changes

winml/lib/Api.Experimental/LearningModelBuilder.cpp Show resolved Hide resolved

martinb35 reviewed Feb 11, 2021

View reviewed changes

Sheil Kumar added 2 commits February 11, 2021 00:32

pr feedback

c1d0831

Merge branch 'master' into user/sheilk/training-explorations

296cf1e

martinb35 approved these changes Feb 12, 2021

View reviewed changes

smk2007 merged commit 87cb6fd into master Feb 12, 2021

smk2007 deleted the user/sheilk/training-explorations branch February 12, 2021 22:17

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add LearningModelBuilder to WinML Experimental Namespace along with various Audio operators#6623

Add LearningModelBuilder to WinML Experimental Namespace along with various Audio operators#6623
smk2007 merged 83 commits intomasterfrom
user/sheilk/training-explorations

smk2007 commented Feb 9, 2021

Uh oh!

Uh oh!

tiagoshibata Feb 10, 2021

Uh oh!

smk2007 Feb 11, 2021

Uh oh!

tiagoshibata Feb 12, 2021

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

martinb35 Feb 11, 2021

Uh oh!

smk2007 Feb 11, 2021

Uh oh!

smk2007 Feb 11, 2021

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

Conversation

smk2007 commented Feb 9, 2021

Uh oh!

Uh oh!

tiagoshibata Feb 10, 2021

Choose a reason for hiding this comment

Uh oh!

smk2007 Feb 11, 2021

Choose a reason for hiding this comment

Uh oh!

tiagoshibata Feb 12, 2021

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

martinb35 Feb 11, 2021

Choose a reason for hiding this comment

Uh oh!

smk2007 Feb 11, 2021

Choose a reason for hiding this comment

Uh oh!

smk2007 Feb 11, 2021

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants